#!/usr/bin/env python3
# -*- coding: UTF-8 -*-
"""
@author: caijj
@version: V1.0
@file: zg_sz_rent_house_detail_page.py
@time: 2022/02/23
"""
import multiprocessing
import os
import re
import uuid
from time import sleep
import pandas as pd
import requests
from fake_useragent import UserAgent
from lxml import etree
from selenium import webdriver
from config.conf import cm
from tools import timeutil
from tools.database_tool import DataBaseTool

time = timeutil.dt_strptime("%Y-%m-%d %H%M%S")


def agent():
    ua = UserAgent(verify_ssl=False)
    headers = {'User-Agent': ua.random}
    return headers


def get_target_url():
    headers = agent()  # 构造请求头
    # headers['Accept'] = {"Content-type": "application/x-www-form-urlencoded", "Accept": "text/plain"}
    headers['Accept'] = 'application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,' \
                        'application/signed-exchange;v=b3;q=0.9 '
    house_detail_link = '//su.zufang.zhuge.com/1003831/3593705.html'
    url = 'https:' + house_detail_link
    res = requests.get(url, headers=headers)
    print(res.text)
    # home_page_doc = pq(url='http://zfcjj.suzhou.gov.cn', headers=headers, encoding='utf-8')
    # home_page_doc = pq(url=url, headers=headers, encoding='utf-8')


def chrome_enter_target_page(url='https://su.xinfang.zhuge.com/'):  # chrome进入目标页面
    chrome_options = webdriver.ChromeOptions()
    # chrome_options.add_argument('--headless')
    # chrome_options.add_argument('--disable-gpu')
    web_driver = webdriver.Chrome(options=chrome_options)
    web_driver.maximize_window()
    web_driver.get(url)
    return web_driver


# def get_rent_house_price_trend(index=0):
#     web_driver = None
#     rent_price_list = []
#     price_trend_list = []
#     quotation_time_list = []
#     rent_house_list = get_rent_house_detail_link()
#     # rent_house_list = [(UUID('749fd08e-908f-11ec-8000-d41b81052116'), '//su.zufang.zhuge.com/1003831/3593705.html')]
#     time = timeutil.dt_strptime("%Y-%m-%d %H%M%S")
#     try:
#         for second_house_info in rent_house_list[index:]:
#             rent_house_id, house_detail_link = second_house_info[0], second_house_info[1]
#             url = 'https:' + house_detail_link
#             web_driver = chrome_enter_target_page(url)
#             sleep(2)
#             html_source = web_driver.page_source  # 该属性可以获取当前浏览器的当前页的源码（html）
#             tree = etree.HTML(html_source)
#             price_parent_ele = tree.xpath(
#                 "//div[@name='navItem']/div[@class='machine_title']/b[contains(text(),'房屋时光机')]")
#             if len(price_parent_ele) != 0:
#                 price_trend_ele_list = tree.xpath("//ul[@class='machine_list']/li")
#                 for price_trend_ele in price_trend_ele_list:
#                     record_id = uuid.uuid4()
#                     ele_index = price_trend_ele_list.index(price_trend_ele)
#                     # time_xpath = "//ul[@class='machine_list']/li[" + str(ele_index + 1) + "]/span[@class='list_date']"
#                     quotation_time = \
#                         tree.xpath(
#                             "//ul[@class='machine_list']/li[" + str(ele_index + 1) + "]/span[@class='list_date']")[
#                             0].text
#                     quotation_time = quotation_time.replace('\n', '')
#                     quotation_time = quotation_time.replace(' ', '')
#                     quotation_time = quotation_time.strip()
#                     quotation_time_list.append(quotation_time)
#                     # now_year = datetime.datetime.now().year
#                     # now_month = datetime.datetime.now().month
#                     # quotation_time_month = quotation_time.split('/')[0]
#                     rent_price = \
#                         tree.xpath(
#                             "//ul[@class='machine_list']/li[" + str(ele_index + 1) + "]/span[@class='list_offer']/p")[
#                             0].text
#                     if len(rent_price.split('：')) > 1:
#                         rent_price = re.findall(r"\d+\.?\d*", rent_price.split('：')[1])[0]
#                     else:
#                         rent_price = re.findall(r"\d+\.?\d*", rent_price.split(': ')[1])[0]
#                     rent_price_list.append(rent_price)
#                     # if len(quotation_time) >= 2:
#                     price_trend_list.append((record_id, rent_house_id, quotation_time, rent_price, time))
#             else:
#                 web_driver.quit()
#             insert_rent_house_price_trend_data(price_trend_list)
#             price_trend_list = []
#             web_driver.quit()
#             index += 1
#     except Exception as e:
#         print(e)
#         print(house_detail_link)
#         web_driver.quit()
#         sleep(10)
#         get_rent_house_price_trend(index)


def get_rent_house_price_trend(house_list, name='诸葛网房价走势数据test.csv', index=0):
    web_driver = None
    time_list, rent_price_list, record_id_list, rent_house_id_list, quotation_time_list = [], [], [], [], []
    # time = timeutil.dt_strptime("%Y-%m-%d %H%M%S")
    try:
        for second_house_info in house_list[index:]:
            rent_house_id, house_detail_link = second_house_info[0], second_house_info[1]
            url = 'https:' + house_detail_link
            url = house_detail_link
            web_driver = chrome_enter_target_page(url)
            sleep(3)
            html_source = web_driver.page_source  # 该属性可以获取当前浏览器的当前页的源码（html）
            tree = etree.HTML(html_source)
            price_parent_ele = tree.xpath(
                "//div[@name='navItem']/div[@class='machine_title']/b[contains(text(),'房屋时光机')]")
            if len(price_parent_ele) != 0:
                price_trend_ele_list = tree.xpath("//ul[@class='machine_list']/li")
                for price_trend_ele in price_trend_ele_list:
                    record_id = uuid.uuid4()
                    ele_index = price_trend_ele_list.index(price_trend_ele)
                    quotation_time = \
                        tree.xpath(
                            "//ul[@class='machine_list']/li[" + str(ele_index + 1) + "]/span[@class='list_date']")[
                            0].text
                    quotation_time = quotation_time.replace('\n', '')
                    quotation_time = quotation_time.replace(' ', '')
                    quotation_time = quotation_time.strip()
                    rent_price = \
                        tree.xpath(
                            "//ul[@class='machine_list']/li[" + str(ele_index + 1) + "]/span[@class='list_offer']/p")[
                            0].text
                    if len(rent_price.split('：')) > 1:
                        rent_price = re.findall(r"\d+\.?\d*", rent_price.split('：')[1])[0]
                    else:
                        rent_price = re.findall(r"\d+\.?\d*", rent_price.split(': ')[1])[0]
                    record_id_list.append(record_id)
                    rent_house_id_list.append(rent_house_id)
                    quotation_time_list.append(quotation_time)
                    rent_price_list.append(rent_price)
                    time_list.append(time)
                data_dict = {'id': record_id_list, 'rent_house_id': rent_house_id_list, 'quotation_time':
                             quotation_time_list, 'rent_price': rent_price_list, 'create_time': time_list}
                df = pd.DataFrame(data_dict)
                time_list, rent_price_list, record_id_list, rent_house_id_list, quotation_time_list = [], [], [], [], []
                # 保存为csv格式
                df.to_csv(cm.tmp_file_path(name), mode="a",
                          columns=['id', 'rent_house_id', 'quotation_time', 'rent_price', 'create_time'],
                          encoding="utf-8",
                          header=False,
                          index=False)
                web_driver.quit()
                index += 1
            else:
                web_driver.quit()
    except Exception as e:
        print(e)
        print(house_detail_link)
        if web_driver is not None:
            web_driver.quit()
        sleep(10)
        get_rent_house_price_trend(house_list, name, index)


def get_rent_house_detail_link(process_count, table_name='mega_zg_sz_zf_estate_info'):
    data_list = []
    sql = "select id,detail_url from {}".format(table_name)
    db = DataBaseTool()
    result = db.select_sql(sql)
    integer = len(result) // 2
    first_half_data = result[:integer]
    step = len(first_half_data) // process_count
    remainder = len(first_half_data) % process_count
    for i in range(1, process_count+2):
        if i == range(1, process_count+2)[-1]:
            data = first_half_data[step*(i-1):]
        else:
            data = first_half_data[step*(i-1):step*i]
        data_list.append(data)
    return data_list


def hexXor(_0x4e08d8, _0x23a392):
    _0x5a5d3b = ''
    _0xe89588 = 0x0
    while _0xe89588 < len(_0x23a392) and _0xe89588 < len(_0x4e08d8):
        _0x401af1 = int(_0x23a392[_0xe89588: _0xe89588 + 0x2], 16)
        _0x105f59 = int(_0x4e08d8[_0xe89588: _0xe89588 + 0x2], 16)
        _0x189e2c = hex(_0x401af1 ^ _0x105f59)
        if len(_0x189e2c) == 0x1:
            _0x189e2c = '\x30' + _0x189e2c
        _0x5a5d3b += _0x189e2c[2:]

        _0xe89588 += 0x2
    return _0x5a5d3b


def unsbox(arg):
    _0x4b082b = [0xf, 0x23, 0x1d, 0x18, 0x21, 0x10, 0x1, 0x26, 0xa, 0x9, 0x13, 0x1f, 0x28, 0x1b, 0x16, 0x17, 0x19, 0xd,
                 0x6, 0xb, 0x27, 0x12, 0x14, 0x8, 0xe, 0x15, 0x20, 0x1a, 0x2, 0x1e, 0x7, 0x4, 0x11, 0x5, 0x3, 0x1c,
                 0x22, 0x25, 0xc, 0x24]
    _0x4da0dc = [''] * 40
    _0x12605e = ''
    for _0x20a7bf in range(0, len(arg)):
        _0x385ee3 = arg[_0x20a7bf]
        for _0x217721 in range(0, len(_0x4b082b)):
            if _0x4b082b[_0x217721] == _0x20a7bf + 0x1:
                _0x4da0dc[_0x217721] = _0x385ee3
    _0x12605e = ''.join(_0x4da0dc)
    return _0x12605e


def get_target_url():
    headers = agent()  # 构造请求头
    headers['Accept'] = 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,' \
                        '*/*;q=0.8,application/signed-exchange;v=b3;q=0.9 '
    # headers['Content-Type'] = 'text/html charset=utf-8'
    # house_detail_link = '//su.zufang.zhuge.com/1010790/3334782.html'
    # proxies = {
    #     "http": "http://127.0.0.1:8888",
    #     "https": "http://127.0.0.1:8888"
    # }
    # url = 'https:' + house_detail_link
    url = 'https://nj.fang.ke.com/loupan/p_hsxlyjlxhtjblkym/'
    # headers['Referer'] = 'http://bj.zhuge.com/p403.html'
    # res = requests.get(url, headers=headers, proxies=proxies, verify=False)
    # # res = requests.get(url, headers=headers, allow_redirects=True, verify=False)
    # print(res)
    # # 重js中匹配出 arg1
    # # arg1 = re.findall("arg1=\'(.*?)\'", res.text)[0]
    # arg1 = '5629175C5186D05FCFC91C60BE783BD56E27259C'
    #
    # # 参数生成
    # key = '3000176000856006061501533003690027800375'
    # _0x23a392 = unsbox(arg1)
    # cookies = {
    #     # "acw_tc": "2760777d16457695181653546eee7e78c419c89f75fdc9fb118f0db895170f",
    #     # 'acw_sc__v2': hexXor(key, _0x23a392)
    #     'acw_sc__v2': '623078351548a7c0bb6d9ecf315d0259e6a8e112'
    # }
    # res = requests.get(url, headers=headers, allow_redirects=True, cookies=cookies, proxies=proxies, verify=False)
    res = requests.get(url, headers=headers, allow_redirects=True)
    print(res.text)


# def get_rent_house_detail_link( table_name='mega_zg_sz_zf_estate_info'):
#     sql = "select id,detail_url from {}".format(table_name)
#     db = DataBaseTool()
#     result = db.select_sql(sql)
#     return result


def insert_rent_house_price_trend_data(data, table_name='mega_zg_sz_zf_price_trend'):
    db = DataBaseTool()
    sql = "insert into {}(id, rent_house_id, quotation_time, rent_price, create_time) values(%s,%s,%s,%s,%s)".format(
        table_name)
    try:
        cursor = db.conn.cursor()
        cursor.executemany(sql, data)
        # cursor.execute(sql, data)
        db.conn.commit()
    except Exception as e:
        print(e)
    db.conn.close()


def del_data(table_name='mega_zg_sz_zf_price_trend'):
    db = DataBaseTool()
    sql = "delete from {}".format(table_name)
    db.update_sql(sql)


def check_file_is_exists(name):
    if os.path.isfile(name):
        os.remove(name)


if __name__ == '__main__':
    # csv_name = '诸葛网房价走势数据test.csv'
    # check_file_is_exists(cm.tmp_file_path(csv_name))
    #
    # count = multiprocessing.cpu_count()
    # pool = multiprocessing.Pool(multiprocessing.cpu_count())
    #
    # rent_house_list = get_rent_house_detail_link(count)
    #
    # pool.map(get_rent_house_price_trend, rent_house_list)
    # pool.close()
    # pool.join()
    get_target_url()

